{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: 定义可以autograd的函数\n",
    "----------------------------------------\n",
    "\n",
    "这里还是那个全连接网络的例子，不过这里我们不使用clamp来实现ReLU，而是我们自己来实现一个MyReLU的函数。\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 31219966.0\n",
      "1 29228598.0\n",
      "2 31855032.0\n",
      "3 33837796.0\n",
      "4 31309610.0\n",
      "5 23219566.0\n",
      "6 14017930.0\n",
      "7 7351454.5\n",
      "8 3824305.25\n",
      "9 2181492.5\n",
      "10 1429416.125\n",
      "11 1053271.25\n",
      "12 838279.4375\n",
      "13 696292.375\n",
      "14 591994.9375\n",
      "15 509986.34375\n",
      "16 443016.09375\n",
      "17 387132.71875\n",
      "18 339843.0\n",
      "19 299528.5\n",
      "20 264898.1875\n",
      "21 235043.203125\n",
      "22 209189.78125\n",
      "23 186678.046875\n",
      "24 167006.03125\n",
      "25 149769.140625\n",
      "26 134602.125\n",
      "27 121227.1484375\n",
      "28 109402.171875\n",
      "29 98908.84375\n",
      "30 89572.1796875\n",
      "31 81244.7421875\n",
      "32 73807.5859375\n",
      "33 67140.8203125\n",
      "34 61165.69140625\n",
      "35 55794.71484375\n",
      "36 50951.6796875\n",
      "37 46584.84375\n",
      "38 42636.40234375\n",
      "39 39065.08984375\n",
      "40 35830.0703125\n",
      "41 32897.5859375\n",
      "42 30235.6875\n",
      "43 27815.34375\n",
      "44 25608.77734375\n",
      "45 23597.3984375\n",
      "46 21764.626953125\n",
      "47 20090.896484375\n",
      "48 18560.94921875\n",
      "49 17160.408203125\n",
      "50 15876.46484375\n",
      "51 14699.072265625\n",
      "52 13618.0888671875\n",
      "53 12626.189453125\n",
      "54 11713.9921875\n",
      "55 10874.2978515625\n",
      "56 10101.07421875\n",
      "57 9388.1953125\n",
      "58 8730.314453125\n",
      "59 8122.8173828125\n",
      "60 7561.619140625\n",
      "61 7042.642578125\n",
      "62 6562.70263671875\n",
      "63 6118.2939453125\n",
      "64 5706.646484375\n",
      "65 5325.21533203125\n",
      "66 4971.63720703125\n",
      "67 4643.28173828125\n",
      "68 4338.45947265625\n",
      "69 4055.298583984375\n",
      "70 3792.241455078125\n",
      "71 3547.55908203125\n",
      "72 3320.00537109375\n",
      "73 3108.183837890625\n",
      "74 2911.02734375\n",
      "75 2727.268310546875\n",
      "76 2556.116943359375\n",
      "77 2396.485107421875\n",
      "78 2247.54638671875\n",
      "79 2108.537353515625\n",
      "80 1978.7432861328125\n",
      "81 1857.5635986328125\n",
      "82 1744.2894287109375\n",
      "83 1638.4476318359375\n",
      "84 1539.4676513671875\n",
      "85 1446.885009765625\n",
      "86 1360.291259765625\n",
      "87 1279.2420654296875\n",
      "88 1203.33154296875\n",
      "89 1132.2841796875\n",
      "90 1065.67529296875\n",
      "91 1003.2247314453125\n",
      "92 944.6898803710938\n",
      "93 889.7554321289062\n",
      "94 838.2390747070312\n",
      "95 789.8998413085938\n",
      "96 744.5142822265625\n",
      "97 701.9252319335938\n",
      "98 661.8890380859375\n",
      "99 624.2869873046875\n",
      "100 588.947021484375\n",
      "101 555.717529296875\n",
      "102 524.4898071289062\n",
      "103 495.10479736328125\n",
      "104 467.477294921875\n",
      "105 441.47119140625\n",
      "106 416.9814453125\n",
      "107 393.932861328125\n",
      "108 372.2206726074219\n",
      "109 351.7697448730469\n",
      "110 332.51043701171875\n",
      "111 314.35589599609375\n",
      "112 297.24566650390625\n",
      "113 281.11920166015625\n",
      "114 265.9047546386719\n",
      "115 251.55967712402344\n",
      "116 238.02749633789062\n",
      "117 225.25387573242188\n",
      "118 213.20701599121094\n",
      "119 201.82955932617188\n",
      "120 191.0914764404297\n",
      "121 180.95352172851562\n",
      "122 171.3895721435547\n",
      "123 162.35040283203125\n",
      "124 153.813720703125\n",
      "125 145.74288940429688\n",
      "126 138.11398315429688\n",
      "127 130.90347290039062\n",
      "128 124.0849609375\n",
      "129 117.63800811767578\n",
      "130 111.53839874267578\n",
      "131 105.76984405517578\n",
      "132 100.31217956542969\n",
      "133 95.14808654785156\n",
      "134 90.25867462158203\n",
      "135 85.63349914550781\n",
      "136 81.25365447998047\n",
      "137 77.10543823242188\n",
      "138 73.17628479003906\n",
      "139 69.45805358886719\n",
      "140 65.93392181396484\n",
      "141 62.596065521240234\n",
      "142 59.43147277832031\n",
      "143 56.434715270996094\n",
      "144 53.59406280517578\n",
      "145 50.90114974975586\n",
      "146 48.34733963012695\n",
      "147 45.92720031738281\n",
      "148 43.63268280029297\n",
      "149 41.45646667480469\n",
      "150 39.39207077026367\n",
      "151 37.433773040771484\n",
      "152 35.57638168334961\n",
      "153 33.812828063964844\n",
      "154 32.14006042480469\n",
      "155 30.55307960510254\n",
      "156 29.046979904174805\n",
      "157 27.617061614990234\n",
      "158 26.25908660888672\n",
      "159 24.969572067260742\n",
      "160 23.7459716796875\n",
      "161 22.58446502685547\n",
      "162 21.48060417175293\n",
      "163 20.432300567626953\n",
      "164 19.436643600463867\n",
      "165 18.4913272857666\n",
      "166 17.592058181762695\n",
      "167 16.738550186157227\n",
      "168 15.927192687988281\n",
      "169 15.156188011169434\n",
      "170 14.42426872253418\n",
      "171 13.72765827178955\n",
      "172 13.065531730651855\n",
      "173 12.436320304870605\n",
      "174 11.838045120239258\n",
      "175 11.26956844329834\n",
      "176 10.728928565979004\n",
      "177 10.214025497436523\n",
      "178 9.725558280944824\n",
      "179 9.260233879089355\n",
      "180 8.817676544189453\n",
      "181 8.397231101989746\n",
      "182 7.996941566467285\n",
      "183 7.61618709564209\n",
      "184 7.25352144241333\n",
      "185 6.9091267585754395\n",
      "186 6.581111431121826\n",
      "187 6.269033432006836\n",
      "188 5.972130298614502\n",
      "189 5.689389705657959\n",
      "190 5.420334339141846\n",
      "191 5.164163589477539\n",
      "192 4.92042350769043\n",
      "193 4.688492774963379\n",
      "194 4.467823505401611\n",
      "195 4.257279872894287\n",
      "196 4.057137489318848\n",
      "197 3.866370916366577\n",
      "198 3.6847620010375977\n",
      "199 3.5119056701660156\n",
      "200 3.3474740982055664\n",
      "201 3.190659523010254\n",
      "202 3.0414040088653564\n",
      "203 2.8988866806030273\n",
      "204 2.7636630535125732\n",
      "205 2.6344261169433594\n",
      "206 2.511683464050293\n",
      "207 2.3945364952087402\n",
      "208 2.282994508743286\n",
      "209 2.176748514175415\n",
      "210 2.075413465499878\n",
      "211 1.9788378477096558\n",
      "212 1.886976718902588\n",
      "213 1.7995249032974243\n",
      "214 1.715970754623413\n",
      "215 1.6365138292312622\n",
      "216 1.5605586767196655\n",
      "217 1.4882639646530151\n",
      "218 1.419459581375122\n",
      "219 1.3536427021026611\n",
      "220 1.2911900281906128\n",
      "221 1.2315386533737183\n",
      "222 1.1746881008148193\n",
      "223 1.1204169988632202\n",
      "224 1.0689036846160889\n",
      "225 1.0195306539535522\n",
      "226 0.9724819660186768\n",
      "227 0.9277315139770508\n",
      "228 0.8850858211517334\n",
      "229 0.8443463444709778\n",
      "230 0.8056568503379822\n",
      "231 0.7686123847961426\n",
      "232 0.7332637906074524\n",
      "233 0.6996608376502991\n",
      "234 0.6674660444259644\n",
      "235 0.6368662714958191\n",
      "236 0.6077296733856201\n",
      "237 0.579842209815979\n",
      "238 0.5532929301261902\n",
      "239 0.5280286073684692\n",
      "240 0.503844678401947\n",
      "241 0.48084113001823425\n",
      "242 0.4587770402431488\n",
      "243 0.43790218234062195\n",
      "244 0.41785845160484314\n",
      "245 0.39883485436439514\n",
      "246 0.3806346654891968\n",
      "247 0.36327090859413147\n",
      "248 0.3466903269290924\n",
      "249 0.33092501759529114\n",
      "250 0.31578895449638367\n",
      "251 0.301440954208374\n",
      "252 0.287708044052124\n",
      "253 0.27460241317749023\n",
      "254 0.26215556263923645\n",
      "255 0.25016990303993225\n",
      "256 0.23885448276996613\n",
      "257 0.22802433371543884\n",
      "258 0.21765680611133575\n",
      "259 0.2077893316745758\n",
      "260 0.19833381474018097\n",
      "261 0.18928895890712738\n",
      "262 0.18069829046726227\n",
      "263 0.1725364476442337\n",
      "264 0.16473372280597687\n",
      "265 0.15725404024124146\n",
      "266 0.15011148154735565\n",
      "267 0.1433171033859253\n",
      "268 0.13684503734111786\n",
      "269 0.1306719034910202\n",
      "270 0.12471942603588104\n",
      "271 0.1190958246588707\n",
      "272 0.11372163146734238\n",
      "273 0.10856622457504272\n",
      "274 0.10369070619344711\n",
      "275 0.09900157153606415\n",
      "276 0.09454187005758286\n",
      "277 0.09028308093547821\n",
      "278 0.08620373904705048\n",
      "279 0.08232803642749786\n",
      "280 0.07860228419303894\n",
      "281 0.07504735141992569\n",
      "282 0.07168334722518921\n",
      "283 0.06842982023954391\n",
      "284 0.06536342948675156\n",
      "285 0.06240985915064812\n",
      "286 0.05958216264843941\n",
      "287 0.05690997466444969\n",
      "288 0.0543552041053772\n",
      "289 0.051920801401138306\n",
      "290 0.04959166795015335\n",
      "291 0.047357022762298584\n",
      "292 0.04523176699876785\n",
      "293 0.0431957021355629\n",
      "294 0.04126520827412605\n",
      "295 0.039434388279914856\n",
      "296 0.037649963051080704\n",
      "297 0.03596176952123642\n",
      "298 0.03435370326042175\n",
      "299 0.032812975347042084\n",
      "300 0.031351350247859955\n",
      "301 0.029942112043499947\n",
      "302 0.02861470729112625\n",
      "303 0.027324534952640533\n",
      "304 0.026113998144865036\n",
      "305 0.024949220940470695\n",
      "306 0.023856328800320625\n",
      "307 0.022787930443882942\n",
      "308 0.021777404472231865\n",
      "309 0.020801639184355736\n",
      "310 0.019883455708622932\n",
      "311 0.01901274174451828\n",
      "312 0.018162859603762627\n",
      "313 0.01735636219382286\n",
      "314 0.016585147008299828\n",
      "315 0.01586228236556053\n",
      "316 0.015155995264649391\n",
      "317 0.014487863518297672\n",
      "318 0.013848425820469856\n",
      "319 0.013244060799479485\n",
      "320 0.012662953697144985\n",
      "321 0.01211503054946661\n",
      "322 0.01158533338457346\n",
      "323 0.01108186412602663\n",
      "324 0.010590325109660625\n",
      "325 0.0101356515660882\n",
      "326 0.009696326218545437\n",
      "327 0.00927804410457611\n",
      "328 0.008878001943230629\n",
      "329 0.00849395152181387\n",
      "330 0.008126414380967617\n",
      "331 0.0077794864773750305\n",
      "332 0.007447630167007446\n",
      "333 0.0071288724429905415\n",
      "334 0.0068205902352929115\n",
      "335 0.00653107138350606\n",
      "336 0.006253038067370653\n",
      "337 0.005988320801407099\n",
      "338 0.00573326600715518\n",
      "339 0.005495618563145399\n",
      "340 0.005261508747935295\n",
      "341 0.005045647732913494\n",
      "342 0.004833341576159\n",
      "343 0.00463094050064683\n",
      "344 0.0044448827393352985\n",
      "345 0.004258272238075733\n",
      "346 0.004085653927177191\n",
      "347 0.003922751639038324\n",
      "348 0.0037573445588350296\n",
      "349 0.0036039953120052814\n",
      "350 0.0034584261011332273\n",
      "351 0.0033198841847479343\n",
      "352 0.003190524410456419\n",
      "353 0.0030607040971517563\n",
      "354 0.002937188372015953\n",
      "355 0.0028218894731253386\n",
      "356 0.0027088397182524204\n",
      "357 0.0026040044613182545\n",
      "358 0.0025007999502122402\n",
      "359 0.002405723789706826\n",
      "360 0.002309610601514578\n",
      "361 0.002224832074716687\n",
      "362 0.0021380605176091194\n",
      "363 0.0020568319596350193\n",
      "364 0.0019767903722822666\n",
      "365 0.001903315307572484\n",
      "366 0.0018304138211533427\n",
      "367 0.001764069776982069\n",
      "368 0.0016999332001432776\n",
      "369 0.0016386212082579732\n",
      "370 0.0015748607693240047\n",
      "371 0.0015184342628344893\n",
      "372 0.0014634114922955632\n",
      "373 0.0014108471805229783\n",
      "374 0.0013596764765679836\n",
      "375 0.0013099127681925893\n",
      "376 0.0012646260438486934\n",
      "377 0.001219899277202785\n",
      "378 0.0011786491377279162\n",
      "379 0.0011357010807842016\n",
      "380 0.001094965380616486\n",
      "381 0.0010584063129499555\n",
      "382 0.001022410811856389\n",
      "383 0.000987407867796719\n",
      "384 0.0009538563317619264\n",
      "385 0.0009229338029399514\n",
      "386 0.0008912050980143249\n",
      "387 0.0008616373525001109\n",
      "388 0.000831698824185878\n",
      "389 0.0008048220770433545\n",
      "390 0.0007799717714078724\n",
      "391 0.000754401262383908\n",
      "392 0.0007297035772353411\n",
      "393 0.0007076715701259673\n",
      "394 0.0006851551006548107\n",
      "395 0.0006619678460992873\n",
      "396 0.0006426713662222028\n",
      "397 0.0006218330236151814\n",
      "398 0.0006030315416865051\n",
      "399 0.0005841052625328302\n",
      "400 0.0005676595610566437\n",
      "401 0.0005495872464962304\n",
      "402 0.0005345349200069904\n",
      "403 0.000518183340318501\n",
      "404 0.0005032058106735349\n",
      "405 0.0004879965854343027\n",
      "406 0.00047303352039307356\n",
      "407 0.00045996089465916157\n",
      "408 0.0004470919957384467\n",
      "409 0.0004344047047197819\n",
      "410 0.0004223627911414951\n",
      "411 0.00041155682993121445\n",
      "412 0.0003989617107436061\n",
      "413 0.0003884337202180177\n",
      "414 0.000376209121895954\n",
      "415 0.00036608503432944417\n",
      "416 0.00035656383261084557\n",
      "417 0.0003466942871455103\n",
      "418 0.00033761223312467337\n",
      "419 0.0003285066341049969\n",
      "420 0.0003199055208824575\n",
      "421 0.00031179070356301963\n",
      "422 0.000303549284581095\n",
      "423 0.000295894977170974\n",
      "424 0.0002886141010094434\n",
      "425 0.000281488464679569\n",
      "426 0.00027399652753956616\n",
      "427 0.0002674937713891268\n",
      "428 0.000261558685451746\n",
      "429 0.00025457286392338574\n",
      "430 0.000248373718932271\n",
      "431 0.00024230143753811717\n",
      "432 0.0002362956729484722\n",
      "433 0.0002309672418050468\n",
      "434 0.00022513976728077978\n",
      "435 0.00022022824850864708\n",
      "436 0.00021410518093034625\n",
      "437 0.00020966977172065526\n",
      "438 0.00020504237909335643\n",
      "439 0.00020002512610517442\n",
      "440 0.0001949847355717793\n",
      "441 0.00019077757315244526\n",
      "442 0.00018637266475707293\n",
      "443 0.0001821288897190243\n",
      "444 0.00017834773461800069\n",
      "445 0.00017383026715833694\n",
      "446 0.0001704621536191553\n",
      "447 0.0001665734889684245\n",
      "448 0.00016279361443594098\n",
      "449 0.0001595132052898407\n",
      "450 0.000155992602230981\n",
      "451 0.00015300672384910285\n",
      "452 0.00015031358634587377\n",
      "453 0.00014661528985016048\n",
      "454 0.00014351523714140058\n",
      "455 0.00014026944700162858\n",
      "456 0.0001375000865664333\n",
      "457 0.00013424841745290905\n",
      "458 0.0001318982831435278\n",
      "459 0.0001289603387704119\n",
      "460 0.0001265643659280613\n",
      "461 0.00012461986625567079\n",
      "462 0.00012205463281134143\n",
      "463 0.00011963544238824397\n",
      "464 0.00011729450488928705\n",
      "465 0.00011486694711493328\n",
      "466 0.00011250937677687034\n",
      "467 0.00011063981946790591\n",
      "468 0.00010876097803702578\n",
      "469 0.0001070971047738567\n",
      "470 0.00010472907888470218\n",
      "471 0.00010262477735523134\n",
      "472 0.00010073527664644644\n",
      "473 9.895420225802809e-05\n",
      "474 9.708665311336517e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "475 9.529116505291313e-05\n",
      "476 9.362120908917859e-05\n",
      "477 9.179205517284572e-05\n",
      "478 9.045530896401033e-05\n",
      "479 8.876148058334365e-05\n",
      "480 8.733809227123857e-05\n",
      "481 8.590223296778277e-05\n",
      "482 8.421840175287798e-05\n",
      "483 8.267716475529596e-05\n",
      "484 8.127454202622175e-05\n",
      "485 7.992322207428515e-05\n",
      "486 7.877988537074998e-05\n",
      "487 7.754105899948627e-05\n",
      "488 7.635498332092538e-05\n",
      "489 7.502910011680797e-05\n",
      "490 7.355350680882111e-05\n",
      "491 7.20532116247341e-05\n",
      "492 7.087919220793992e-05\n",
      "493 6.966372893657535e-05\n",
      "494 6.870318611618131e-05\n",
      "495 6.756203947588801e-05\n",
      "496 6.656658661086112e-05\n",
      "497 6.57599521218799e-05\n",
      "498 6.4848420151975e-05\n",
      "499 6.390204362105578e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class MyReLU(torch.autograd.Function):\n",
    "    \"\"\"\n",
    "    为了实现自定义的实现autograd的函数，我们需要基础torch.autograd.Function，\n",
    "    然后再实现forward和backward两个函数。\n",
    "    \"\"\"\n",
    "\n",
    "    @staticmethod\n",
    "    def forward(ctx, input):\n",
    "        \"\"\"\n",
    "        在forward函数，我们的输入是input，然后我们根据input计算输出。同时为了下面的backward，\n",
    "        我们需要使用save\\_for\\_backward来保存用于反向计算的数据到ctx里，这里我们需要保存input。\n",
    "        \"\"\"\n",
    "        ctx.save_for_backward(input)\n",
    "        return input.clamp(min=0)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        \"\"\"\n",
    "        从ctx.saved\\_tensors里恢复input\n",
    "        然后用input计算梯度\n",
    "        \"\"\"\n",
    "        input, = ctx.saved_tensors\n",
    "        grad_input = grad_output.clone()\n",
    "        grad_input[input < 0] = 0\n",
    "        return grad_input\n",
    "\n",
    "\n",
    "dtype = torch.float\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "x = torch.randn(N, D_in, device=device, dtype=dtype)\n",
    "y = torch.randn(N, D_out, device=device, dtype=dtype)\n",
    "\n",
    "w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n",
    "w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # 为了调用我们自定义的函数，我们需要使用Function.apply方法，把它命名为'relu'\n",
    "    relu = MyReLU.apply\n",
    "\n",
    "    # 我们使用自定义的ReLU来进行Forward计算\n",
    "    y_pred = relu(x.mm(w1)).mm(w2)\n",
    " \n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    " \n",
    "    loss.backward()\n",
    " \n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    " \n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
